from google.colab import drive
drive.mount('/content/drive')

Mounted at /content/drive

import pandas as pd
import numpy as np

import matplotlib.pyplot as plt
import seaborn as sns

# To scale the data using z-score
from sklearn.preprocessing import StandardScaler

from sklearn.model_selection import train_test_split

# Algorithms to use
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis

from sklearn.linear_model import LogisticRegression

from sklearn.neighbors import KNeighborsClassifier

# Metrics to evaluate the model
from sklearn.metrics import confusion_matrix, classification_report, precision_recall_curve

# For tuning the model
from sklearn.model_selection import GridSearchCV

# To ignore warnings
import warnings
warnings.filterwarnings("ignore")

sns.set()

# Reading the dataset
df = pd.read_csv('/content/drive/MyDrive/My DS DA/Employee Attrition/data.csv')

df.head()

df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1470 entries, 0 to 1469
Data columns (total 35 columns):
 #   Column                    Non-Null Count  Dtype 
---  ------                    --------------  ----- 
 0   Age                       1470 non-null   int64 
 1   Attrition                 1470 non-null   object
 2   BusinessTravel            1470 non-null   object
 3   DailyRate                 1470 non-null   int64 
 4   Department                1470 non-null   object
 5   DistanceFromHome          1470 non-null   int64 
 6   Education                 1470 non-null   int64 
 7   EducationField            1470 non-null   object
 8   EmployeeCount             1470 non-null   int64 
 9   EmployeeNumber            1470 non-null   int64 
 10  EnvironmentSatisfaction   1470 non-null   int64 
 11  Gender                    1470 non-null   object
 12  HourlyRate                1470 non-null   int64 
 13  JobInvolvement            1470 non-null   int64 
 14  JobLevel                  1470 non-null   int64 
 15  JobRole                   1470 non-null   object
 16  JobSatisfaction           1470 non-null   int64 
 17  MaritalStatus             1470 non-null   object
 18  MonthlyIncome             1470 non-null   int64 
 19  MonthlyRate               1470 non-null   int64 
 20  NumCompaniesWorked        1470 non-null   int64 
 21  Over18                    1470 non-null   object
 22  OverTime                  1470 non-null   object
 23  PercentSalaryHike         1470 non-null   int64 
 24  PerformanceRating         1470 non-null   int64 
 25  RelationshipSatisfaction  1470 non-null   int64 
 26  StandardHours             1470 non-null   int64 
 27  StockOptionLevel          1470 non-null   int64 
 28  TotalWorkingYears         1470 non-null   int64 
 29  TrainingTimesLastYear     1470 non-null   int64 
 30  WorkLifeBalance           1470 non-null   int64 
 31  YearsAtCompany            1470 non-null   int64 
 32  YearsInCurrentRole        1470 non-null   int64 
 33  YearsSinceLastPromotion   1470 non-null   int64 
 34  YearsWithCurrManager      1470 non-null   int64 
dtypes: int64(26), object(9)
memory usage: 402.1+ KB

df.nunique()

df.columns.to_list()

['Age',
 'Attrition',
 'BusinessTravel',
 'DailyRate',
 'Department',
 'DistanceFromHome',
 'Education',
 'EducationField',
 'EmployeeCount',
 'EmployeeNumber',
 'EnvironmentSatisfaction',
 'Gender',
 'HourlyRate',
 'JobInvolvement',
 'JobLevel',
 'JobRole',
 'JobSatisfaction',
 'MaritalStatus',
 'MonthlyIncome',
 'MonthlyRate',
 'NumCompaniesWorked',
 'Over18',
 'OverTime',
 'PercentSalaryHike',
 'PerformanceRating',
 'RelationshipSatisfaction',
 'StandardHours',
 'StockOptionLevel',
 'TotalWorkingYears',
 'TrainingTimesLastYear',
 'WorkLifeBalance',
 'YearsAtCompany',
 'YearsInCurrentRole',
 'YearsSinceLastPromotion',
 'YearsWithCurrManager']

df = df.drop(['EmployeeNumber', 'Over18', 'StandardHours', 'EmployeeCount'], axis = 1)

# Creating numerical columns
num_cols = ['Age',
 'DailyRate',
 'DistanceFromHome',
 'Education',
 'EnvironmentSatisfaction',
 'HourlyRate',
 'JobInvolvement',
 'JobLevel',
 'JobSatisfaction',
 'MonthlyIncome',
 'MonthlyRate',
 'NumCompaniesWorked',
 'PercentSalaryHike',
 'PerformanceRating',
 'RelationshipSatisfaction',
 'StockOptionLevel',
 'TotalWorkingYears',
 'TrainingTimesLastYear',
 'WorkLifeBalance',
 'YearsAtCompany',
 'YearsInCurrentRole',
 'YearsSinceLastPromotion',
 'YearsWithCurrManager']

# Creating categorical variables
cat_cols = ['Attrition',
 'BusinessTravel',
 'Department',
 'EducationField',
 'Gender',
 'JobRole',
 'MaritalStatus',
 'OverTime']

df.isnull().sum().sum()

np.int64(0)

df.duplicated().sum()

np.int64(0)

# Checking summary statistics
df[num_cols].describe().T

# Creating histograms
df[num_cols].hist(figsize = (14,14))
plt.tight_layout()

for i in cat_cols:
  print(df[i].value_counts(normalize = True))

  print('*' * 40)

Attrition
No     0.838776
Yes    0.161224
Name: proportion, dtype: float64
****************************************
BusinessTravel
Travel_Rarely        0.709524
Travel_Frequently    0.188435
Non-Travel           0.102041
Name: proportion, dtype: float64
****************************************
Department
Research & Development    0.653741
Sales                     0.303401
Human Resources           0.042857
Name: proportion, dtype: float64
****************************************
EducationField
Life Sciences       0.412245
Medical             0.315646
Marketing           0.108163
Technical Degree    0.089796
Other               0.055782
Human Resources     0.018367
Name: proportion, dtype: float64
****************************************
Gender
Male      0.6
Female    0.4
Name: proportion, dtype: float64
****************************************
JobRole
Sales Executive              0.221769
Research Scientist           0.198639
Laboratory Technician        0.176190
Manufacturing Director       0.098639
Healthcare Representative    0.089116
Manager                      0.069388
Sales Representative         0.056463
Research Director            0.054422
Human Resources              0.035374
Name: proportion, dtype: float64
****************************************
MaritalStatus
Married     0.457823
Single      0.319728
Divorced    0.222449
Name: proportion, dtype: float64
****************************************
OverTime
No     0.717007
Yes    0.282993
Name: proportion, dtype: float64
****************************************

#Crosstab
pd.crosstab(df['OverTime'], df['Attrition'], normalize = 'index')

# How many employees do not work over time and have not left the company

df[(df['OverTime'] == 'No') & (df['Attrition'] == 'No')].shape

(944, 31)

# #mployes who do not work over time but still left the company

df[(df['OverTime'] == 'No') & (df['Attrition'] == 'Yes')].shape

(110, 31)

# Proportion of employees who do not work overtime, but still left the company
110/(944 + 110)

0.10436432637571158

fig, axes = plt.subplots(nrows=2, ncols=4, figsize=(15, 10), sharey=True)

i = 0
for x in range(2):
    for y in range(4):
        if i < len(cat_cols):
            pd.crosstab(df[cat_cols[i]], df['Attrition'], normalize='index')\
              .mul(100).plot(kind='bar', stacked=True, ax=axes[x, y])
            axes[x, y].set_ylabel('Percentage Attrition %')
            axes[x, y].set_title(cat_cols[i])
            i += 1
        else:
            axes[x, y].axis('off')  # Hide empty subplot

plt.tight_layout()
plt.show()

# The mean of numerical variables grouped by attrition
df.groupby(['Attrition'])[num_cols].mean()

# Plotting the correlation between numerical variables
plt.figure(figsize = (15, 8))

mask = np.triu(df[num_cols].corr())
sns.heatmap(df[num_cols].corr(), annot = True, fmt = '0.2f', cmap = 'YlGnBu', mask=mask)

<Axes: >

#replace values in a pandas DataFrame

#Method 1
#df['OverTime'].replace({'Yes': 1, 'No': 0})

#Method 2
np.where(df['OverTime'] == 'Yes', 1, 0)
  # This checks for each value in the OverTime column if it is equal to 'Yes'
  # If the condition is True, it assigns 1, otherwise 0 is assigned

array([1, 0, 1, ..., 1, 0, 0])

# Creating dummy variables for categorical Variables

# Creating the list of columns for which we need to create the dummy variables
to_get_dummies_for = ['BusinessTravel', 'Department', 'Education', 'EducationField', 'EnvironmentSatisfaction', 'Gender', 'JobInvolvement',
                      'JobLevel', 'JobRole', 'MaritalStatus']

# Creating dummy variables
df = pd.get_dummies(data = df, columns = to_get_dummies_for, drop_first = True)

# Mapping overtime and attrition
dict_OverTime = {'Yes': 1, 'No': 0} # just like above (created to map categorical string values ('Yes' and 'No') into numeric values (1 and 0)
dict_attrition = {'Yes': 1, 'No': 0}

df['OverTime'] = df.OverTime.map(dict_OverTime) # maps the 'OverTime' column's values ('Yes' to 1, 'No' to 0).
df['Attrition'] = df.Attrition.map(dict_attrition)

# Separating the independent variables (X) and the dependent variable (Y)

# Separating the target variable and other variables
Y = df.Attrition
X = df.drop(columns = ['Attrition']) #target

X.shape # training data

(1470, 54)

data = df.copy()

from sklearn.preprocessing import (
    StandardScaler, MinMaxScaler, RobustScaler, MaxAbsScaler,
    PowerTransformer, QuantileTransformer, Normalizer
)

# 📌 Load dataset (Modify path if needed)
#file_path = "your_dataset.csv"  # Update with correct file
#data = pd.read_csv(file_path)

# 📌 Select numeric columns only
numeric_features = data.select_dtypes(include=['number']).dropna()

# 📌 Detect Outliers using IQR (Interquartile Range)
def detect_outliers(df1):
    outlier_info = {}
    for column in df1.columns:
        Q1 = df1[column].quantile(0.25)
        Q3 = df1[column].quantile(0.75)
        IQR = Q3 - Q1
        lower_bound = Q1 - 1.5 * IQR
        upper_bound = Q3 + 1.5 * IQR

        # Count outliers
        outlier_count = ((df1[column] < lower_bound) | (df1[column] > upper_bound)).sum()
        outlier_info[column] = outlier_count

    return outlier_info

outliers = detect_outliers(numeric_features)
outlier_df1 = pd.DataFrame.from_dict(outliers, orient='index', columns=['Outlier Count'])

# 📌 Check Skewness (to determine if transformation is needed)
skewness = numeric_features.skew()

# 📌 Scaling Options with All Variants
scalers = {
    "StandardScaler": StandardScaler(),
    "MinMaxScaler": MinMaxScaler(),
    "RobustScaler": RobustScaler(),
    "MaxAbsScaler": MaxAbsScaler(),
    "PowerTransformer (Yeo-Johnson)": PowerTransformer(method="yeo-johnson"),
    "PowerTransformer (Box-Cox)": PowerTransformer(method="box-cox"),
    "QuantileTransformer (Normal)": QuantileTransformer(output_distribution="normal"),
    "QuantileTransformer (Uniform)": QuantileTransformer(output_distribution="uniform"),
    "Normalizer (L1)": Normalizer(norm="l1"),
    "Normalizer (L2)": Normalizer(norm="l2"),
    "Normalizer (Max)": Normalizer(norm="max"),
}

scaler_results = {}

# 📌 Try all scalers & transformations
for scaler_name, scaler in scalers.items():
    try:
        transformed_data = scaler.fit_transform(numeric_features)
        transformed_df1 = pd.DataFrame(transformed_data, columns=numeric_features.columns, index=numeric_features.index)

        # Calculate variance after scaling (Higher is better)
        cumulative_variance = transformed_df1.var().sum()
        scaler_results[scaler_name] = cumulative_variance

    except Exception as e:
        print(f"❌ Error with {scaler_name}: {e}")

# 📌 Convert results to DataFrame and Rank
scaler_rank_df1 = pd.DataFrame(list(scaler_results.items()), columns=['Scaler', 'Cumulative Variance'])
scaler_rank_df1 = scaler_rank_df1.sort_values(by="Cumulative Variance", ascending=False)

# 📌 Pick the best scaler based on variance ranking
best_scaler_name = scaler_rank_df1.iloc[0]["Scaler"]
best_scaler = scalers[best_scaler_name]
scaled_data = best_scaler.fit_transform(numeric_features)

# 📌 Convert Scaled Data to DataFrame
scaled_df1 = pd.DataFrame(scaled_data, columns=numeric_features.columns, index=numeric_features.index)

# 📌 Save the scaled dataset
scaled_df1.to_csv("/content/drive/MyDrive/My DS DA/Employee Attrition/scaled_dataset.csv", index=False)

# 📌 Explain Selection
explanations = {
    "MinMaxScaler": "Scales data between [0,1]. Used in Unsupervised Learning & Deep Learning but NOT recommended for outliers.",
    "StandardScaler": "Standardizes data to mean=0, variance=1. Recommended for Unsupervised Learning & Deep Learning. Works well with outliers.",
    "RobustScaler": "Uses median and IQR. Best choice for handling outliers in both Unsupervised and Supervised Learning.",
    "MaxAbsScaler": "Scales data between [-1,1]. Used for sparse data, not generally needed for typical datasets.",
    "PowerTransformer (Yeo-Johnson)": "Transforms data to be more Gaussian-like. Good for skewed data in Regression & Classification.",
    "PowerTransformer (Box-Cox)": "Similar to Yeo-Johnson but works only with strictly positive values. Reduces skewness.",
    "QuantileTransformer (Normal)": "Maps data to a normal distribution. Works well when distribution is unknown.",
    "QuantileTransformer (Uniform)": "Maps data to a uniform distribution. Useful for highly irregular distributions.",
    "Normalizer (L1)": "Normalizes each row by L1 norm. Good for text-based or sparse datasets.",
    "Normalizer (L2)": "Normalizes each row by L2 norm. Often used in clustering tasks.",
    "Normalizer (Max)": "Normalizes each row by its maximum absolute value. Good for text and sparse data."
}

# 📌 Print Summary
print("\n📊 Scaler Rankings:")
print(scaler_rank_df1)

print(f"\n🏆 Best Scaler Chosen: {best_scaler_name}")
print(f"📌 Reason: {explanations.get(best_scaler_name, 'No explanation available')}")

# 📌 Print Insights
if outlier_df1['Outlier Count'].sum() > 0:
    print("\n⚠️ Outliers detected! **RobustScaler** is recommended if handling them is critical.")
else:
    print("\n✅ No significant outliers detected. **StandardScaler** is the default recommendation.")

if any(abs(skewness) > 1):
    print("\n⚠️ Skewed data detected! **PowerTransformer**")

❌ Error with PowerTransformer (Box-Cox): The Box-Cox transformation can only be applied to strictly positive data

📊 Scaler Rankings:
                           Scaler  Cumulative Variance
5    QuantileTransformer (Normal)           135.841283
0                  StandardScaler            21.014295
4  PowerTransformer (Yeo-Johnson)            20.013615
2                    RobustScaler            11.679481
6   QuantileTransformer (Uniform)             2.197743
1                    MinMaxScaler             1.695346
3                    MaxAbsScaler             1.284149
9                Normalizer (Max)             0.135088
8                 Normalizer (L2)             0.098395
7                 Normalizer (L1)             0.060545

🏆 Best Scaler Chosen: QuantileTransformer (Normal)
📌 Reason: Maps data to a normal distribution. Works well when distribution is unknown.

⚠️ Outliers detected! **RobustScaler** is recommended if handling them is critical.

⚠️ Skewed data detected! **PowerTransformer**

# # Scaling the data
# sc = StandardScaler()

# X_scaled = sc.fit_transform(X)

# X_scaled = pd.DataFrame(X_scaled, columns = X.columns)

from sklearn.preprocessing import QuantileTransformer

# Scaling the data
sc = QuantileTransformer(output_distribution='normal')

X_scaled = sc.fit_transform(X)

X_scaled = pd.DataFrame(X_scaled, columns=X.columns)

# Splitting the data
x_train, x_test, y_train, y_test = train_test_split(X_scaled, Y, test_size = 0.3, random_state = 1, stratify = Y)

sns.countplot(data=df, x='Attrition', edgecolor = "black");

# Number of samples in the target variable. Original data
sum(Y == 0), sum(Y == 1)

(1233, 237)

# Proportion of class "0"
round(2466/(2466+474) * 100, 2)

83.88

# Number of samples in the target variable. Training data after split using "stratify"
sum(y_train == 0), sum(y_train == 1)

(863, 166)

# Proportion of class "0"
round(1726/(1726+332) * 100 ,2)

83.87

# Number of samples in the target variable. Training data after split without using "stratify"

_, _, y_train_no_strat, y_test_no_strat = train_test_split(X_scaled, Y, test_size = 0.3, random_state=1)

sum(y_train_no_strat == 0), sum(y_train_no_strat == 1)

(869, 160)

# Proportion of class "0"
round(1737/(1737 + 321) *100, 2)

84.4

def metrics_score(actual, predicted):

    print(classification_report(actual, predicted))

    # Confusion matrix
    cm = confusion_matrix(actual, predicted)

    # Plot confusion matrix
    plt.figure(figsize = (7, 4))

    sns.heatmap(cm, annot = True, fmt = '.2f', xticklabels = ['Not Attrite', 'Attrite'], yticklabels = ['Not Attrite', 'Attrite'])

    plt.ylabel('Actual')
    plt.xlabel('Predicted')

    plt.show()

def model_performance_classification(model, predictors, target):
    """
    Function to compute different metrics to check classification model performance

    model: classifier

    predictors: independent variables

    target: dependent variable
    """

    # Predicting using the independent variables
    pred = model.predict(predictors)

    recall = recall_score(target, pred,average = 'macro')                 # To compute recall

    precision = precision_score(target, pred, average = 'macro')              # To compute precision

    acc = accuracy_score(target, pred)                                 # To compute accuracy score


    # Creating a dataframe of metrics

    df_perf = pd.DataFrame(
        {
            "Precision":  precision,
            "Recall":  recall,
            "Accuracy": acc,
        },

        index = [0],
    )

    return df_perf

from sklearn.tree import DecisionTreeClassifier

# Building decision tree model
# dt = DecisionTreeClassifier(class_weight = {0: 0.17, 1: 0.83}, random_state = 1)

dt_no_weights = DecisionTreeClassifier(random_state = 1)

dt_weights = DecisionTreeClassifier(class_weight = {0: 0.17, 1: 0.83}, random_state = 1) # to give more importance to the class with the lowest percentage
# class weight (hyperparameter) is useful when dealing with unbalanced/imbalanced data: see cell 74 above (sns.countplot(data=df, x='Attrition', edgecolor = 'black')

# cell 50: Percentage of Label 0 (Not Attriate): 83.88. Percentage of Label 1 (Attritiate): 16.12 = 0.83 given to class with lowest percentage = class 1 !!!!!!!!!!!
  # flipped/inversed it
  # for unbalanced/imbalanced data
  # can be optimized but instructor stated better to get form distribution (cell 74)

# # Fitting decision tree model
# dt.fit(x_train, y_train)

# Fitting decision tree model
dt_no_weights.fit(x_train, y_train)

DecisionTreeClassifier(random_state=1)

DecisionTreeClassifier(random_state=1)

# Fitting decision tree model
dt_weights.fit(x_train, y_train)

DecisionTreeClassifier(class_weight={0: 0.17, 1: 0.83}, random_state=1)

DecisionTreeClassifier(class_weight={0: 0.17, 1: 0.83}, random_state=1)

# Checking performance on the training dataset
y_train_pred_dt = dt_weights.predict(x_train)

metrics_score(y_train, y_train_pred_dt)

              precision    recall  f1-score   support

           0       1.00      1.00      1.00       863
           1       1.00      1.00      1.00       166

    accuracy                           1.00      1029
   macro avg       1.00      1.00      1.00      1029
weighted avg       1.00      1.00      1.00      1029

# Checking performance on the test dataset
y_test_pred_dt = dt_weights.predict(x_test)

metrics_score(y_test, y_test_pred_dt)

              precision    recall  f1-score   support

           0       0.88      0.85      0.86       370
           1       0.34      0.41      0.37        71

    accuracy                           0.78       441
   macro avg       0.61      0.63      0.62       441
weighted avg       0.79      0.78      0.78       441

# performance_classification with training data

model_performance_classification(dt_weights, x_train, y_train)

# dtree_test = model_performance_classification(dt,x_test,y_test)
dtree_test = model_performance_classification(dt_weights,x_test,y_test)
dtree_test

importances = dt_weights.feature_importances_
importances

array([3.79936593e-02, 6.82162389e-02, 8.10343094e-02, 5.09446909e-02,
       4.39879277e-02, 1.01489495e-01, 6.73985155e-02, 1.13790091e-02,
       6.06921082e-02, 2.03138490e-02, 1.08217662e-15, 2.01208561e-02,
       8.23041683e-02, 3.20681971e-02, 1.67164389e-02, 1.22094076e-02,
       6.53640377e-02, 1.70946813e-02, 1.57082809e-02, 1.43828341e-02,
       1.97231699e-16, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
       7.67886455e-03, 0.00000000e+00, 2.19606906e-03, 0.00000000e+00,
       1.72890644e-02, 0.00000000e+00, 1.16721790e-17, 7.71928139e-03,
       9.95701923e-03, 2.54090092e-02, 2.19240018e-02, 1.38341564e-02,
       1.26714609e-02, 1.05208205e-02, 0.00000000e+00, 1.96886075e-03,
       0.00000000e+00, 5.80786172e-03, 1.67413487e-03, 0.00000000e+00,
       3.81392623e-03, 8.66106155e-03, 0.00000000e+00, 3.62580696e-03,
       3.45506258e-03, 1.16862326e-02, 3.47014779e-04, 1.03415854e-02,
       0.00000000e+00, 0.00000000e+00])

# Plot the feature importance

importances = dt_weights.feature_importances_

columns = X.columns

importance_df = pd.DataFrame(importances, index = columns, columns = ['Importance']).sort_values(by = 'Importance', ascending = False)

plt.figure(figsize = (13, 13))

sns.barplot(x=importance_df.Importance,y=importance_df.index, palette = 'rocket');

# from sklearn import metrics

# %%time
# # Choose the type of classifier
# dtree_tuned = DecisionTreeClassifier(class_weight = {0: 0.17, 1: 0.83}, random_state = 1)

# # Grid of parameters to choose from
# parameters = {'max_depth': np.arange(2, 7), # should test a wide range of values 2, 15, 50, 100
#               'criterion': ['gini', 'entropy'],
#               'min_samples_leaf': [5, 10, 20, 25]}

# # Type of scoring used to compare parameter combinations.
# # "pos_label": It allows you to specify which class label should be considered as the positive class when calculating the scoring metric.
# # By default, pos_label = 1, but you can change it based on your specific use case.
# scorer = metrics.make_scorer(recall_score, pos_label = 1)

# # Grid search object
# gridCV = GridSearchCV(dtree_tuned, parameters, scoring = scorer, cv = 10) # cross validation, GridSearchCV to tune hyperparameters

# # Fitting the grid search on the train data
# gridCV = gridCV.fit(x_train, y_train)

# # Set the classifier to the best combination of parameters
# dtree_tuned = gridCV.best_estimator_

# # Fit the best estimator to the data
# dtree_tuned.fit(x_train, y_train)

from sklearn import metrics
from sklearn.metrics import recall_score

# Choose the type of classifier
dtree_tuned = DecisionTreeClassifier(class_weight={0: 0.17, 1: 0.83}, random_state=1)

# Grid of parameters to choose from
parameters = {
    'max_depth': np.arange(2, 7),
    'criterion': ['gini', 'entropy'],
    'min_samples_leaf': [5, 10, 20, 25]
}

# Type of scoring used to compare parameter combinations
scorer = metrics.make_scorer(recall_score, pos_label=1)

# Grid search object
gridCV = GridSearchCV(dtree_tuned, parameters, scoring=scorer, cv=10)

# Fit the grid search on the train data
gridCV = gridCV.fit(x_train, y_train)

# Set the classifier to the best combination of parameters
dtree_tuned = gridCV.best_estimator_

# Fit the best estimator to the data
dtree_tuned.fit(x_train, y_train)

DecisionTreeClassifier(class_weight={0: 0.17, 1: 0.83}, max_depth=np.int64(2),
                       min_samples_leaf=5, random_state=1)

DecisionTreeClassifier(class_weight={0: 0.17, 1: 0.83}, max_depth=np.int64(2),
                       min_samples_leaf=5, random_state=1)

# Show attributes in the object
gridCV.best_estimator_.__dict__

{'criterion': 'gini',
 'splitter': 'best',
 'max_depth': np.int64(2),
 'min_samples_split': 2,
 'min_samples_leaf': 5,
 'min_weight_fraction_leaf': 0.0,
 'max_features': None,
 'max_leaf_nodes': None,
 'random_state': 1,
 'min_impurity_decrease': 0.0,
 'class_weight': {0: 0.17, 1: 0.83},
 'ccp_alpha': 0.0,
 'monotonic_cst': None,
 'feature_names_in_': array(['Age', 'DailyRate', 'DistanceFromHome', 'HourlyRate',
        'JobSatisfaction', 'MonthlyIncome', 'MonthlyRate',
        'NumCompaniesWorked', 'OverTime', 'PercentSalaryHike',
        'PerformanceRating', 'RelationshipSatisfaction',
        'StockOptionLevel', 'TotalWorkingYears', 'TrainingTimesLastYear',
        'WorkLifeBalance', 'YearsAtCompany', 'YearsInCurrentRole',
        'YearsSinceLastPromotion', 'YearsWithCurrManager',
        'BusinessTravel_Travel_Frequently', 'BusinessTravel_Travel_Rarely',
        'Department_Research & Development', 'Department_Sales',
        'Education_2', 'Education_3', 'Education_4', 'Education_5',
        'EducationField_Life Sciences', 'EducationField_Marketing',
        'EducationField_Medical', 'EducationField_Other',
        'EducationField_Technical Degree', 'EnvironmentSatisfaction_2',
        'EnvironmentSatisfaction_3', 'EnvironmentSatisfaction_4',
        'Gender_Male', 'JobInvolvement_2', 'JobInvolvement_3',
        'JobInvolvement_4', 'JobLevel_2', 'JobLevel_3', 'JobLevel_4',
        'JobLevel_5', 'JobRole_Human Resources',
        'JobRole_Laboratory Technician', 'JobRole_Manager',
        'JobRole_Manufacturing Director', 'JobRole_Research Director',
        'JobRole_Research Scientist', 'JobRole_Sales Executive',
        'JobRole_Sales Representative', 'MaritalStatus_Married',
        'MaritalStatus_Single'], dtype=object),
 'n_features_in_': 54,
 'n_outputs_': 1,
 'classes_': array([0, 1]),
 'n_classes_': np.int64(2),
 'max_features_': 54,
 'tree_': <sklearn.tree._tree.Tree at 0x7d97502b30c0>}

# Checking performance on the TRAINING DATASET
y_train_pred_dt = dtree_tuned.predict(x_train)

metrics_score(y_train, y_train_pred_dt)

              precision    recall  f1-score   support

           0       0.91      0.63      0.74       863
           1       0.25      0.66      0.37       166

    accuracy                           0.63      1029
   macro avg       0.58      0.64      0.56      1029
weighted avg       0.80      0.63      0.68      1029

# Checking performance on the TEST SET
y_test_pred_dt = dtree_tuned.predict(x_test)

metrics_score(y_test, y_test_pred_dt)

              precision    recall  f1-score   support

           0       0.89      0.57      0.70       370
           1       0.22      0.63      0.33        71

    accuracy                           0.58       441
   macro avg       0.56      0.60      0.51       441
weighted avg       0.78      0.58      0.64       441

from sklearn.metrics import precision_score, recall_score, accuracy_score

# TEST DATA
dtree_tuned_test = model_performance_classification(dtree_tuned,x_test,y_test)
dtree_tuned_test

temp = pd.concat([model_performance_classification(dt_weights, x_train, y_train),
                  model_performance_classification(dt_weights, x_test, y_test)], axis=0)

temp.index = ['Training dataset', 'Test dataset']

temp

temp = pd.concat([model_performance_classification(dtree_tuned, x_train, y_train),
                  model_performance_classification(dtree_tuned, x_test, y_test)], axis=0)

temp.index = ['Training dataset', 'Test dataset']

temp

# Feature importance
importances = dtree_tuned.feature_importances_

# Rename columns
columns = X.columns

importance_df = pd.DataFrame(importances, index = columns, columns = ['Importance']).sort_values(by = 'Importance', ascending = False)

# Plot
plt.figure(figsize = (13, 13))
sns.barplot(x= importance_df.Importance, y= importance_df.index);

from sklearn import tree

features = list(X.columns)

plt.figure(figsize = (15, 10), dpi=300)

# "node_ids": Show the ID number on each node.
# "class_names": Names of each of the target classes in ascending numerical order
tree.plot_tree(dt_no_weights, max_depth = 2, feature_names = features, filled = True, fontsize = 12, node_ids = True, class_names = True)

plt.show()

# Number of training samples
x_train.shape[0]

1029

# Number of samples of label 0 (No attrition)
y_train[y_train == 0].shape[0]

863

# Number of samples of label 1 (Attrition)
y_train[y_train == 1].shape[0]

166

# Number of samples where feature "OverTime" is lower than or equal to 0.5
x_train[x_train.OverTime <= 0.5].shape[0]

757

# Number of samples where feature "OverTime" is greater than 0.5
x_train[x_train.OverTime > 0.5].shape[0]

272

757 + 272

1029

features = list(X.columns)

plt.figure(figsize = (22, 12), dpi=300)

tree.plot_tree(dt_weights, max_depth = 3, feature_names = features, filled = True, fontsize = 12, node_ids = True, class_names = True)

plt.show()

features = list(X.columns)

plt.figure(figsize = (15, 10), dpi=300)

tree.plot_tree(dtree_tuned, max_depth = 4, feature_names = features, filled = True, fontsize = 12, node_ids = True, class_names = True)

plt.show()

from sklearn.ensemble import RandomForestClassifier

# Fitting the Random Forest classifier on the training data
rf_estimator = RandomForestClassifier(class_weight = {0: 0.17, 1: 0.83}, random_state = 1)

rf_estimator.fit(x_train, y_train)

RandomForestClassifier(class_weight={0: 0.17, 1: 0.83}, random_state=1)

RandomForestClassifier(class_weight={0: 0.17, 1: 0.83}, random_state=1)

# Checking performance on the training data
y_pred_train_rf = rf_estimator.predict(x_train)

metrics_score(y_train, y_pred_train_rf)

              precision    recall  f1-score   support

           0       1.00      1.00      1.00       863
           1       1.00      1.00      1.00       166

    accuracy                           1.00      1029
   macro avg       1.00      1.00      1.00      1029
weighted avg       1.00      1.00      1.00      1029

# Checking performance on the testing data
y_pred_test_rf = rf_estimator.predict(x_test)

metrics_score(y_test, y_pred_test_rf)

              precision    recall  f1-score   support

           0       0.85      0.99      0.92       370
           1       0.78      0.10      0.17        71

    accuracy                           0.85       441
   macro avg       0.81      0.55      0.55       441
weighted avg       0.84      0.85      0.80       441

# Checking performance on the TRAINING SET
model_performance_classification(rf_estimator, x_train, y_train)

# Checking performance on the TEST SET
rf_estimator_test = model_performance_classification(rf_estimator,x_test,y_test)
rf_estimator_test

importances = rf_estimator.feature_importances_

columns = X.columns

importance_df = pd.DataFrame(importances, index = columns, columns = ['Importance']).sort_values(by = 'Importance', ascending = False)

plt.figure(figsize = (13, 13))

sns.barplot(x= importance_df.Importance, y=importance_df.index);

%%time
# Choose the type of classifier
rf_estimator_tuned = RandomForestClassifier(class_weight = {0: 0.17, 1: 0.83}, random_state = 1)

# Grid of parameters to choose from
params_rf = {"n_estimators": [100, 250, 500],
             "min_samples_leaf": np.arange(1, 4, 1),
             "max_features": [0.7, 0.9, 'auto']} # max features: number of features to randomly select from data set (.7 is 70% of features to 90%)

# Type of scoring used to compare parameter combinations - recall score for class 1
scorer = metrics.make_scorer(recall_score, pos_label = 1)

# Run the grid search
grid_obj = GridSearchCV(rf_estimator_tuned, params_rf, scoring = scorer, cv = 5)

grid_obj = grid_obj.fit(x_train, y_train)

# Set the classifier to the best combination of parameters
rf_estimator_tuned = grid_obj.best_estimator_

CPU times: user 3min 11s, sys: 271 ms, total: 3min 11s
Wall time: 3min 34s

rf_estimator_tuned.fit(x_train, y_train)

RandomForestClassifier(class_weight={0: 0.17, 1: 0.83}, max_features=0.7,
                       min_samples_leaf=np.int64(3), n_estimators=250,
                       random_state=1)

RandomForestClassifier(class_weight={0: 0.17, 1: 0.83}, max_features=0.7,
                       min_samples_leaf=np.int64(3), n_estimators=250,
                       random_state=1)

# Checking performance on the training data
y_pred_train_rf_tuned = rf_estimator_tuned.predict(x_train)

metrics_score(y_train, y_pred_train_rf_tuned)

              precision    recall  f1-score   support

           0       1.00      1.00      1.00       863
           1       0.99      1.00      1.00       166

    accuracy                           1.00      1029
   macro avg       1.00      1.00      1.00      1029
weighted avg       1.00      1.00      1.00      1029

# Checking performance on the test data
y_pred_test_rf_tuned = rf_estimator_tuned.predict(x_test)

metrics_score(y_test, y_pred_test_rf_tuned)

              precision    recall  f1-score   support

           0       0.89      0.96      0.92       370
           1       0.62      0.37      0.46        71

    accuracy                           0.86       441
   macro avg       0.75      0.66      0.69       441
weighted avg       0.84      0.86      0.85       441

# Checking performance on the TRAINING SET
model_performance_classification(rf_estimator_tuned, x_train, y_train)

# Checking performance on the TEST SET
rf_estimator_tuned_test = model_performance_classification(rf_estimator_tuned, x_test, y_test)
rf_estimator_tuned_test

# Plotting feature importance
importances = rf_estimator_tuned.feature_importances_

columns = X.columns

importance_df = pd.DataFrame(importances, index = columns, columns = ['Importance']).sort_values(by = 'Importance', ascending = False)

plt.figure(figsize = (13, 13))

sns.barplot(x= importance_df.Importance, y= importance_df.index)

<Axes: xlabel='Importance', ylabel='None'>

# # Installing the xgboost library using the 'pip' command.
# !pip install xgboost

# Importing the AdaBoostClassifier and GradientBoostingClassifier [Boosting]
from sklearn.ensemble import AdaBoostClassifier, GradientBoostingClassifier

# Importing the XGBClassifier from the xgboost library
from xgboost import XGBClassifier

# Adaboost Classifier
adaboost_model = AdaBoostClassifier(random_state = 1)

# Fitting the model
adaboost_model.fit(x_train, y_train)

# Model Performance on the test data
adaboost_model_perf_test = model_performance_classification(adaboost_model,x_test,y_test)

adaboost_model_perf_test

# Gradient Boost Classifier
gbc = GradientBoostingClassifier(random_state = 1)

# Fitting the model
gbc.fit(x_train, y_train)

# Model Performance on the test data
gbc_perf_test = model_performance_classification(gbc, x_test, y_test)

gbc_perf_test

# XGBoost Classifier
xgb = XGBClassifier(random_state = 1, eval_metric = 'logloss')

# Fitting the model
xgb.fit(x_train,y_train)

# Model Performance on the test data
xgb_perf_test = model_performance_classification(xgb,x_test,y_test)

xgb_perf_test

models_test_comp_df = pd.concat([dtree_test.T,
                                 dtree_tuned_test.T,
                                 rf_estimator_test.T,
                                 rf_estimator_tuned_test.T,
                                 adaboost_model_perf_test.T,
                                 gbc_perf_test.T,
                                 xgb_perf_test.T], axis = 1)

models_test_comp_df.columns = ["Decision Tree classifier",
                               "Tuned Decision Tree classifier",
                               "Random Forest classifier",
                               "Tuned Random Forest classifier",
                               "Adaboost classifier",
                               "Gradientboost classifier",
                               "XGBoost classifier"]

print("Test performance comparison:")

Test performance comparison:

models_test_comp_df

notebook_path = '/content/drive/MyDrive/My DS DA/Employee Attrition/Employee Attrition Prediction.ipynb'

!jupyter nbconvert --to html "{notebook_path}"

from google.colab import files
files.download('/content/drive/MyDrive/My DS DA/Employee Attrition/Employee Attrition Prediction.html')

	count	mean	std	min	25%	50%	75%	max
Age	1470.0	36.923810	9.135373	18.0	30.0	36.0	43.00	60.0
DailyRate	1470.0	802.485714	403.509100	102.0	465.0	802.0	1157.00	1499.0
DistanceFromHome	1470.0	9.192517	8.106864	1.0	2.0	7.0	14.00	29.0
Education	1470.0	2.912925	1.024165	1.0	2.0	3.0	4.00	5.0
EnvironmentSatisfaction	1470.0	2.721769	1.093082	1.0	2.0	3.0	4.00	4.0
HourlyRate	1470.0	65.891156	20.329428	30.0	48.0	66.0	83.75	100.0
JobInvolvement	1470.0	2.729932	0.711561	1.0	2.0	3.0	3.00	4.0
JobLevel	1470.0	2.063946	1.106940	1.0	1.0	2.0	3.00	5.0
JobSatisfaction	1470.0	2.728571	1.102846	1.0	2.0	3.0	4.00	4.0
MonthlyIncome	1470.0	6502.931293	4707.956783	1009.0	2911.0	4919.0	8379.00	19999.0
MonthlyRate	1470.0	14313.103401	7117.786044	2094.0	8047.0	14235.5	20461.50	26999.0
NumCompaniesWorked	1470.0	2.693197	2.498009	0.0	1.0	2.0	4.00	9.0
PercentSalaryHike	1470.0	15.209524	3.659938	11.0	12.0	14.0	18.00	25.0
PerformanceRating	1470.0	3.153741	0.360824	3.0	3.0	3.0	3.00	4.0
RelationshipSatisfaction	1470.0	2.712245	1.081209	1.0	2.0	3.0	4.00	4.0
StockOptionLevel	1470.0	0.793878	0.852077	0.0	0.0	1.0	1.00	3.0
TotalWorkingYears	1470.0	11.279592	7.780782	0.0	6.0	10.0	15.00	40.0
TrainingTimesLastYear	1470.0	2.799320	1.289271	0.0	2.0	3.0	3.00	6.0
WorkLifeBalance	1470.0	2.761224	0.706476	1.0	2.0	3.0	3.00	4.0
YearsAtCompany	1470.0	7.008163	6.126525	0.0	3.0	5.0	9.00	40.0
YearsInCurrentRole	1470.0	4.229252	3.623137	0.0	2.0	3.0	7.00	18.0
YearsSinceLastPromotion	1470.0	2.187755	3.222430	0.0	0.0	1.0	3.00	15.0
YearsWithCurrManager	1470.0	4.123129	3.568136	0.0	2.0	3.0	7.00	17.0

	Age	DailyRate	DistanceFromHome	Education	EnvironmentSatisfaction	HourlyRate	JobInvolvement	JobLevel	JobSatisfaction	MonthlyIncome	...	PerformanceRating	RelationshipSatisfaction	StockOptionLevel	TotalWorkingYears	TrainingTimesLastYear	WorkLifeBalance	YearsAtCompany	YearsInCurrentRole	YearsSinceLastPromotion	YearsWithCurrManager
Attrition
No	37.561233	812.504461	8.915653	2.927007	2.771290	65.952149	2.770479	2.145985	2.778589	6832.739659	...	3.153285	2.733982	0.845093	11.862936	2.832928	2.781022	7.369019	4.484185	2.234388	4.367397
Yes	33.607595	750.362869	10.632911	2.839662	2.464135	65.573840	2.518987	1.637131	2.468354	4787.092827	...	3.156118	2.599156	0.527426	8.244726	2.624473	2.658228	5.130802	2.902954	1.945148	2.852321

	Precision	Recall	Accuracy
Training dataset	1.00000	1.000000	1.00000
Test dataset	0.60945	0.627198	0.77551

	Precision	Recall	Accuracy
Training dataset	0.579915	0.643493	0.634597
Test dataset	0.556216	0.603388	0.582766

Model Type	Dataset	Precision	Recall	Accuracy
RF Untuned	Training	1.000	1.000	1.000
	Test	0.814	0.547	0.850
RF Tuned (GridSearch)	Training	0.997	0.999	0.999
	Test	0.753	0.661	0.862

IBM HR Employee Attrition¶

Context¶

Objective¶

Dataset Description¶

Importing Liberies¶

Loading the dataset¶

Data Overview¶

Info¶

Unique Values¶

Exploratory Data Analysis¶

Univariate analysis of numerical columns¶

Univariate analysis for categorical variables¶

Bivariate and Multivariate analysis¶

Summary of EDA¶

Model Building - Approach¶

Preparing data for modeling¶

Scaling Optoins and Ranking¶

Scaling the data¶

Splitting the data into 70% train and 30% test sets¶

Check for Imbalanced Data¶

Model evaluation criterion¶

Building the model¶

Building a Decision Tree Model¶

Tuning the Decision Tree Classifier using GridSearch¶

Random Forest¶

Tuning the Random Forest classifier using GridSearch¶

Boosting Models¶

XGBoost¶

Hyperparameter Tuning: Boosting¶

Conclusion¶

Column	Description
Age	Age of the employee
Attrition	Whether the employee left the company (Yes/No)
BusinessTravel	Frequency of business travel
DailyRate	Daily salary rate
Department	Department the employee belongs to
DistanceFromHome	Distance from employee's home to workplace
Education	Education level (1–5)
EducationField	Field of education (e.g., Life Sciences, Marketing)
EnvironmentSatisfaction	Satisfaction with work environment (1–4)
Gender	Employee gender
HourlyRate	Hourly wage
JobInvolvement	Level of job involvement (1–4)
JobLevel	Employee job level (1–5)
JobRole	Specific job title
JobSatisfaction	Satisfaction with the job (1–4)
MaritalStatus	Marital status
MonthlyIncome	Monthly salary
MonthlyRate	Monthly rate
NumCompaniesWorked	Number of companies worked for prior
OverTime	Whether employee works overtime (Yes/No)
PercentSalaryHike	Percentage salary increase
PerformanceRating	Performance rating (1–4)
RelationshipSatisfaction	Satisfaction with relationships (1–4)
StockOptionLevel	Stock option level
TotalWorkingYears	Total years of professional experience
TrainingTimesLastYear	Times participated in training in last year
WorkLifeBalance	Work-life balance rating (1–4)
YearsAtCompany	Years spent at the company
YearsInCurrentRole	Years in the current role
YearsSinceLastPromotion	Years since last promotion
YearsWithCurrManager	Years under current manager

	Age	Attrition	BusinessTravel	DailyRate	Department	DistanceFromHome	Education	EducationField	EmployeeCount	EmployeeNumber	...	RelationshipSatisfaction	StandardHours	StockOptionLevel	TotalWorkingYears	TrainingTimesLastYear	WorkLifeBalance	YearsAtCompany	YearsInCurrentRole	YearsSinceLastPromotion	YearsWithCurrManager
0	41	Yes	Travel_Rarely	1102	Sales	1	2	Life Sciences	1	1	...	1	80	0	8	0	1	6	4	0	5
1	49	No	Travel_Frequently	279	Research & Development	8	1	Life Sciences	1	2	...	4	80	1	10	3	3	10	7	1	7
2	37	Yes	Travel_Rarely	1373	Research & Development	2	2	Other	1	4	...	2	80	0	7	3	3	0	0	0	0
3	33	No	Travel_Frequently	1392	Research & Development	3	4	Life Sciences	1	5	...	3	80	0	8	3	3	8	7	3	0
4	27	No	Travel_Rarely	591	Research & Development	2	1	Medical	1	7	...	4	80	1	6	3	3	2	2	2	2

Metric	Train Set	Test Set
Accuracy	1.00	0.78
Precision (Attrite)	1.00	0.34
Recall (Attrite)	1.00	0.41
F1-score (Attrite)	1.00	0.37
Macro Avg F1	1.00	0.63
Weighted Avg F1	1.00	0.78

Model	Dataset	Precision (1)	Recall (1)	F1-Score (1)	Accuracy
Decision Tree (Weighted)	Train	1.00	1.00	1.00	1.00
	Test	0.34	0.41	0.37	0.78
DT Tuned (GridSearchCV)	Train	0.25	0.66	0.37	0.65
	Test	0.22	0.63	0.32	0.64
Random Forest (Weighted)	Train	1.00	1.00	1.00	1.00
	Test	0.10	0.78	0.17	0.85

	Decision Tree classifier	Tuned Decision Tree classifier	Random Forest classifier	Tuned Random Forest classifier	Adaboost classifier	Gradientboost classifier	XGBoost classifier
Precision	0.609450	0.556216	0.814815	0.753133	0.808492	0.789661	0.786274
Recall	0.627198	0.603388	0.546593	0.661477	0.598877	0.648458	0.666882
Accuracy	0.775510	0.582766	0.850340	0.861678	0.861678	0.868481	0.870748

Attrition	No	Yes
OverTime
No	0.895636	0.104364
Yes	0.694712	0.305288